package UrlExtractor;

import types.Movie;


import imdb_db.Imdb;

import java.io.*;
import java.net.*;
import java.util.*;


import omdb.GetMovieInfo;

import org.apache.commons.lang3.*;


/**
 * this class get a list of movie names from 2000 to 2014 which will be used in training set.
 * @author chenji
 *
 */
public class GetMovieNames {
	public static List<String> readFromFile(String fileName) throws Exception{
		File file = new File("movies");
		FileInputStream in = new FileInputStream(file);
		BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
		String line = "";
		List<String> list = new LinkedList<String>();
		while ( (line = reader.readLine()) != null) {
			String id = line.substring(0, line.indexOf(","));
			list.add(id);
		}
		return list;
	}
	static String replaceHtmlUrlEncoding(String s) {
		String[] encode = { "&quot;:%22", "&#x22;:%22", // '"' -> %22
				"&#039;:'", "&#x27;:'", // '
				"&#xE9;:\u00e9", //
				"&#xCA;:\u00ca", //
				" :+", // ' ' -> +
				"&amp;:%26", "&#038;:%26", "&#x26;:%26", "&:%26" }; // '&' -> %26
		for (String e : encode) {
			s = s.replace(e.substring(0, e.indexOf(':')), e.substring(e.indexOf(':') + 1));
		}
		return s;
	}

	static String readDir(String link) {
		String dir = "";
		URL obj;
		try {
			obj = new URL(link);
			HttpURLConnection con = (HttpURLConnection) obj.openConnection();
			con.setRequestMethod("GET");
			con.setRequestProperty("User-Agent", "Mozilla/5.0");

			BufferedReader reader = new BufferedReader(new InputStreamReader(con.getInputStream(), "UTF-8"));
			String line;
			while ((line = reader.readLine()) != null) {
				if (line.contains("Directed By:")) {
					line = line.substring(line.indexOf("Directed By") + "Directed By: </b>".length());
					dir = line.substring(line.indexOf(">") + 1, line.indexOf("</span>"));
					dir = dir.trim();
					break;
				}
			}
			reader.close();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			System.out.println(link);
			e.printStackTrace();
			return "Dir: Not found!";
		}
		if (dir.length() == 0)
			return "Dir: Not found!";
		return StringEscapeUtils.unescapeHtml4(dir);
	}

	public static List<Movie> getMovieNamesFrom(String url, int year) throws Exception {
		List<Movie> movies = new ArrayList<Movie>();
		URL obj = new URL(url);
		HttpURLConnection con = (HttpURLConnection) obj.openConnection();
		con.setRequestMethod("GET");
		con.setRequestProperty("User-Agent", "Mozilla/5.0");

		BufferedReader reader = new BufferedReader(new InputStreamReader(con.getInputStream(), "UTF-8"));
		String line;
		String matrix = null;
		while ((line = reader.readLine()) != null) {
			// System.out.println(line);
			if (line.indexOf("movie-matrix") != -1) {
				matrix = line;
				while ((line = reader.readLine()) != null) {
					if (line.indexOf("</div>") != -1) {
						break;
					}
					matrix += line;
				}
			}
		}
		reader.close();
		// System.out.println(matrix);
		String[] splits = matrix.split("><");
		String title;
		String link;
		String dir;
		for (String s : splits) {
			if (s.startsWith("a href")) {
				link = s.substring(8, s.indexOf("\" alt="));
				// System.out.println(link);
				dir = readDir(link);
				// System.out.println(dir);
				title = s.substring(s.indexOf("\" alt=") + 7);
				title = title.substring(0, title.length() - 2);
				title = title.substring(0, title.indexOf("movie poste") - 1);
				// System.out.println(title + ": " +
				// StringEscapeUtils.unescapeHtml4(title));
				title = StringEscapeUtils.unescapeHtml4(title.trim());
				if (title.matches(".*\\([^\\)]*\\)$")) {
					title = title.substring(0, title.lastIndexOf("(")).trim();
				}
				// System.out.println(title);
				Movie movie = new Movie(title, dir, year);
				// movie.set(GetMovieInfo.sendGet(title));
				movies.add(movie);
			}
		}
		System.out.println(movies.size());
		return movies;
	}

	public static List<Movie> getMovieNamesFrom2(String url, int year) throws Exception {
		List<Movie> movies = new ArrayList<Movie>();
		URL obj = new URL(url);
		HttpURLConnection con = (HttpURLConnection) obj.openConnection();
		con.setRequestMethod("GET");
		con.setRequestProperty("User-Agent", "Mozilla/5.0");

		BufferedReader reader = new BufferedReader(new InputStreamReader(con.getInputStream()));
		String line;
		String title;
		String link;
		String dir;
		String s = "";
		while ((line = reader.readLine()) != null) {
			// System.out.println(line);
			if (line.contains("<a class=\"post-grid")) {
				s += line;
				while ((line = reader.readLine()) != null) {
					s += line;
					if (line.contains("</a")) {
						break;
					}
				}
				link = s.substring(s.indexOf("href=") + 6, s.indexOf("\">")).trim();
				// System.out.println(link);
				dir = readDir(link);
				title = s.substring(s.indexOf("<h3>") + 4);
				title = title.substring(0, title.indexOf("</h3>"));
				title = StringEscapeUtils.unescapeHtml4(title.trim());
				if (title.matches(".*\\([^\\)]*\\)$")) {
					title = title.substring(0, title.lastIndexOf("(")).trim();
				}
				// System.out.println(title);
				Movie movie = new Movie(title, dir, year);
				// movie.set(GetMovieInfo.sendGet(title));
				movies.add(movie);
				s = line.contains("<a class=\"post-grid") ? line : "";
			}
		}
		reader.close();
		System.out.println(movies.size());
		return movies;
	}

	public static void main(String args[]) throws Exception {
		List<Movie> list = new ArrayList<Movie>();
		list.addAll(GetMovieNames.getMovieNamesFrom(
				"http://www.wildaboutmovies.com/2014_movies/Movies_In_Theaters_2014.html", 2014));
		list.addAll(GetMovieNames.getMovieNamesFrom(
				"http://www.wildaboutmovies.com/2013_movies/Movies_In_Theaters_2013.html", 2013));
		list.addAll(GetMovieNames.getMovieNamesFrom2("http://www.wildaboutmovies.com/2012_movies/", 2012));
		list.addAll(GetMovieNames.getMovieNamesFrom2("http://www.wildaboutmovies.com/2011_movies/", 2011));
		list.addAll(GetMovieNames.getMovieNamesFrom2("http://www.wildaboutmovies.com/2010_movies/", 2010));
		list.addAll(GetMovieNames.getMovieNamesFrom2("http://www.wildaboutmovies.com/2009_movies/", 2009));
		list.addAll(GetMovieNames.getMovieNamesFrom2("http://www.wildaboutmovies.com/2008_movies/", 2008));
		list.addAll(GetMovieNames.getMovieNamesFrom2("http://www.wildaboutmovies.com/2007_movies/", 2007));
		list.addAll(GetMovieNames.getMovieNamesFrom2("http://www.wildaboutmovies.com/2006_movies/", 2006));
		System.out.println(list.size());
		File file = new File("movies");
		FileOutputStream out = new FileOutputStream(file, true);
		BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
		for (Movie movie : list) {
			boolean found = false;
			for (Movie m : Imdb.sendGet(URLEncoder.encode(movie.getTitle(), "UTF-8"))) {
				if (movie.getDir().toLowerCase().contains(m.getDir().toLowerCase())) {
					// System.out.println("find: " + m.toString());
					found = true;
					writer.write(m.toString() + "\n");
				}
			}
			if (!found) {
				System.out.println("cant find: " + movie.getTitle() + ", Dir:" + movie.getDir());
			}
		}
		writer.flush();
		writer.close();
	}
}
